{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Feature Creation: Combine with reference feature\n",
    "\n",
    "The CombineWithReferenceFeature() applies combines a group of variables with a group of reference variables utilising mathematical operations ['sub', 'div','add','mul'], returning one or more additional features as a result.\n",
    "\n",
    "For this demonstration, we use the UCI Wine Quality Dataset.\n",
    "\n",
    "The data is publicly available on [UCI repository](https://archive.ics.uci.edu/ml/datasets/Wine+Quality)\n",
    "\n",
    "P. Cortez, A. Cerdeira, F. Almeida, T. Matos and J. Reis. Modeling wine preferences by data mining from physicochemical properties. In Decision Support Systems, Elsevier, 47(4):547-553, 2009."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score,\n",
    "    roc_curve,\n",
    "    roc_auc_score,\n",
    "    classification_report,\n",
    "    confusion_matrix,\n",
    ")\n",
    "from sklearn.pipeline import Pipeline as pipe\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from feature_engine.creation import CombineWithReferenceFeature\n",
    "from feature_engine.imputation import MeanMedianImputer\n",
    "\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0.098</td>\n",
       "      <td>25.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.9968</td>\n",
       "      <td>3.20</td>\n",
       "      <td>0.68</td>\n",
       "      <td>9.8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.04</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0.092</td>\n",
       "      <td>15.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>3.26</td>\n",
       "      <td>0.65</td>\n",
       "      <td>9.8</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.2</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.56</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.075</td>\n",
       "      <td>17.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.9980</td>\n",
       "      <td>3.16</td>\n",
       "      <td>0.58</td>\n",
       "      <td>9.8</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0            7.4              0.70         0.00             1.9      0.076   \n",
       "1            7.8              0.88         0.00             2.6      0.098   \n",
       "2            7.8              0.76         0.04             2.3      0.092   \n",
       "3           11.2              0.28         0.56             1.9      0.075   \n",
       "4            7.4              0.70         0.00             1.9      0.076   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \\\n",
       "0                 11.0                  34.0   0.9978  3.51       0.56   \n",
       "1                 25.0                  67.0   0.9968  3.20       0.68   \n",
       "2                 15.0                  54.0   0.9970  3.26       0.65   \n",
       "3                 17.0                  60.0   0.9980  3.16       0.58   \n",
       "4                 11.0                  34.0   0.9978  3.51       0.56   \n",
       "\n",
       "   alcohol  quality  \n",
       "0      9.4        5  \n",
       "1      9.8        5  \n",
       "2      9.8        5  \n",
       "3      9.8        6  \n",
       "4      9.4        5  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read data\n",
    "data = pd.read_csv('winequality-red.csv', sep=';')\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This Data contains 11 features, all numerical, with no missing values.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quality_range</th>\n",
       "      <th>quality</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  quality_range  quality\n",
       "0             0        5\n",
       "1             0        5\n",
       "2             0        5\n",
       "3             1        6\n",
       "4             0        5"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's transform the Target, i.e Wine Quality into a binary classification problem:\n",
    "\n",
    "bins = [0,5,10]\n",
    "\n",
    "labels = [0, 1] # 'low'=0, 'high'=1\n",
    "\n",
    "data['quality_range']= pd.cut(x=data['quality'], bins=bins, labels=labels)\n",
    "\n",
    "data[['quality_range','quality']].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop original target\n",
    "\n",
    "data.drop('quality', axis=1, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sub and Div Combinators:\n",
    "\n",
    "Let's create two new variables:\n",
    "\n",
    "- non_free_sulfur_dioxide = total sulfur dioxide - free sulfur dioxide\n",
    "- percentage_free_sulfur = free sulfur dioxide / total sulfur dioxide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the Combinators\n",
    "\n",
    "# this transformer substracts free sulfur from total sulfur\n",
    "sub_with_reference_feature = CombineWithReferenceFeature(\n",
    "    variables_to_combine=['total sulfur dioxide'],\n",
    "    reference_variables=['free sulfur dioxide'],\n",
    "    operations=['sub'],\n",
    "    new_variables_names=['non_free_sulfur_dioxide']\n",
    ")\n",
    "\n",
    "# this transformer divides free sulfur by total sulfur\n",
    "div_with_reference_feature = CombineWithReferenceFeature(\n",
    "    variables_to_combine=['free sulfur dioxide'],\n",
    "    reference_variables=['total sulfur dioxide'],\n",
    "    operations=['div'],\n",
    "    new_variables_names=['percentage_free_sulfur']\n",
    ")\n",
    "\n",
    "# Fit the Sub Combinator on training data\n",
    "sub_with_reference_feature.fit(data)\n",
    "\n",
    "# perform the substraction\n",
    "data_t = sub_with_reference_feature.transform(data)\n",
    "\n",
    "# perform division\n",
    "# We can combine both steps in a single call with \".fit_transform()\" method\n",
    "data_t = div_with_reference_feature.fit_transform(data_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality_range</th>\n",
       "      <th>non_free_sulfur_dioxide</th>\n",
       "      <th>percentage_free_sulfur</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.323529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0.098</td>\n",
       "      <td>25.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.9968</td>\n",
       "      <td>3.20</td>\n",
       "      <td>0.68</td>\n",
       "      <td>9.8</td>\n",
       "      <td>0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>0.373134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.04</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0.092</td>\n",
       "      <td>15.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>3.26</td>\n",
       "      <td>0.65</td>\n",
       "      <td>9.8</td>\n",
       "      <td>0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0.277778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.2</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.56</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.075</td>\n",
       "      <td>17.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.9980</td>\n",
       "      <td>3.16</td>\n",
       "      <td>0.58</td>\n",
       "      <td>9.8</td>\n",
       "      <td>1</td>\n",
       "      <td>43.0</td>\n",
       "      <td>0.283333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.323529</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0            7.4              0.70         0.00             1.9      0.076   \n",
       "1            7.8              0.88         0.00             2.6      0.098   \n",
       "2            7.8              0.76         0.04             2.3      0.092   \n",
       "3           11.2              0.28         0.56             1.9      0.075   \n",
       "4            7.4              0.70         0.00             1.9      0.076   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \\\n",
       "0                 11.0                  34.0   0.9978  3.51       0.56   \n",
       "1                 25.0                  67.0   0.9968  3.20       0.68   \n",
       "2                 15.0                  54.0   0.9970  3.26       0.65   \n",
       "3                 17.0                  60.0   0.9980  3.16       0.58   \n",
       "4                 11.0                  34.0   0.9978  3.51       0.56   \n",
       "\n",
       "   alcohol quality_range  non_free_sulfur_dioxide  percentage_free_sulfur  \n",
       "0      9.4             0                     23.0                0.323529  \n",
       "1      9.8             0                     42.0                0.373134  \n",
       "2      9.8             0                     39.0                0.277778  \n",
       "3      9.8             1                     43.0                0.283333  \n",
       "4      9.4             0                     23.0                0.323529  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note the additional variables at the end of the dataframe\n",
    "\n",
    "data_t.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Combine with more than 1 operation\n",
    "\n",
    "We can also combine the variables with more than 1 mathematical operation. And the transformer has the option to create variable names automatically.\n",
    "\n",
    "Here we will create the following variables:\n",
    "\n",
    "- ratio_fixed_to_volatile_acidity = fixed acidity / volatile acidity\n",
    "- total_acidity = fixed acidity + volatile acidity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the Combinator\n",
    "\n",
    "multiple_combinator = CombineWithReferenceFeature(\n",
    "    variables_to_combine=['fixed acidity'],\n",
    "    reference_variables=['volatile acidity'],\n",
    "    operations=['div', 'add'],\n",
    "    new_variables_names=['ratio_fixed_to_volatile', 'total_acidity']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CombineWithReferenceFeature(new_variables_names=['ratio_fixed_to_volatile',\n",
       "                                                 'total_acidity'],\n",
       "                            operations=['div', 'add'],\n",
       "                            reference_variables=['volatile acidity'],\n",
       "                            variables_to_combine=['fixed acidity'])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the Combinator to the training data\n",
    "\n",
    "multiple_combinator.fit(data_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform the data\n",
    "\n",
    "data_t = multiple_combinator.transform(data_t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fixed acidity</th>\n",
       "      <th>volatile acidity</th>\n",
       "      <th>citric acid</th>\n",
       "      <th>residual sugar</th>\n",
       "      <th>chlorides</th>\n",
       "      <th>free sulfur dioxide</th>\n",
       "      <th>total sulfur dioxide</th>\n",
       "      <th>density</th>\n",
       "      <th>pH</th>\n",
       "      <th>sulphates</th>\n",
       "      <th>alcohol</th>\n",
       "      <th>quality_range</th>\n",
       "      <th>non_free_sulfur_dioxide</th>\n",
       "      <th>percentage_free_sulfur</th>\n",
       "      <th>ratio_fixed_to_volatile</th>\n",
       "      <th>total_acidity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.323529</td>\n",
       "      <td>10.571429</td>\n",
       "      <td>8.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.6</td>\n",
       "      <td>0.098</td>\n",
       "      <td>25.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>0.9968</td>\n",
       "      <td>3.20</td>\n",
       "      <td>0.68</td>\n",
       "      <td>9.8</td>\n",
       "      <td>0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>0.373134</td>\n",
       "      <td>8.863636</td>\n",
       "      <td>8.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.8</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.04</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0.092</td>\n",
       "      <td>15.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>0.9970</td>\n",
       "      <td>3.26</td>\n",
       "      <td>0.65</td>\n",
       "      <td>9.8</td>\n",
       "      <td>0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0.277778</td>\n",
       "      <td>10.263158</td>\n",
       "      <td>8.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.2</td>\n",
       "      <td>0.28</td>\n",
       "      <td>0.56</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.075</td>\n",
       "      <td>17.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.9980</td>\n",
       "      <td>3.16</td>\n",
       "      <td>0.58</td>\n",
       "      <td>9.8</td>\n",
       "      <td>1</td>\n",
       "      <td>43.0</td>\n",
       "      <td>0.283333</td>\n",
       "      <td>40.000000</td>\n",
       "      <td>11.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7.4</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.9</td>\n",
       "      <td>0.076</td>\n",
       "      <td>11.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>0.9978</td>\n",
       "      <td>3.51</td>\n",
       "      <td>0.56</td>\n",
       "      <td>9.4</td>\n",
       "      <td>0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>0.323529</td>\n",
       "      <td>10.571429</td>\n",
       "      <td>8.10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \\\n",
       "0            7.4              0.70         0.00             1.9      0.076   \n",
       "1            7.8              0.88         0.00             2.6      0.098   \n",
       "2            7.8              0.76         0.04             2.3      0.092   \n",
       "3           11.2              0.28         0.56             1.9      0.075   \n",
       "4            7.4              0.70         0.00             1.9      0.076   \n",
       "\n",
       "   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \\\n",
       "0                 11.0                  34.0   0.9978  3.51       0.56   \n",
       "1                 25.0                  67.0   0.9968  3.20       0.68   \n",
       "2                 15.0                  54.0   0.9970  3.26       0.65   \n",
       "3                 17.0                  60.0   0.9980  3.16       0.58   \n",
       "4                 11.0                  34.0   0.9978  3.51       0.56   \n",
       "\n",
       "   alcohol quality_range  non_free_sulfur_dioxide  percentage_free_sulfur  \\\n",
       "0      9.4             0                     23.0                0.323529   \n",
       "1      9.8             0                     42.0                0.373134   \n",
       "2      9.8             0                     39.0                0.277778   \n",
       "3      9.8             1                     43.0                0.283333   \n",
       "4      9.4             0                     23.0                0.323529   \n",
       "\n",
       "   ratio_fixed_to_volatile  total_acidity  \n",
       "0                10.571429           8.10  \n",
       "1                 8.863636           8.68  \n",
       "2                10.263158           8.56  \n",
       "3                40.000000          11.48  \n",
       "4                10.571429           8.10  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note the additional variables at the end of the dataframe\n",
    "\n",
    "data_t.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pipeline Example\n",
    "\n",
    "We can put all these transformations into single pipeline:\n",
    "\n",
    "Create new variables scale features and train a Logistic Regression model to predict the wine quality range.\n",
    "\n",
    "See more on how to use Feature-engine within Scikit-learn Pipelines in these [examples](https://github.com/solegalli/feature_engine/tree/master/examples/Pipelines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1439, 11), (160, 11))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = data.drop(['quality_range'], axis=1)\n",
    "\n",
    "y = data.quality_range\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size=0.1,\n",
    "                                                    random_state=0,\n",
    "                                                    shuffle=True,\n",
    "                                                    stratify=y\n",
    "                                                    )\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "value_pipe = pipe([\n",
    "    \n",
    "    # Create new features\n",
    "    ('subtraction', CombineWithReferenceFeature(\n",
    "        variables_to_combine=['total sulfur dioxide'],\n",
    "        reference_variables=['free sulfur dioxide'],\n",
    "        operations=['sub'],\n",
    "        new_variables_names=['non_free_sulfur_dioxide']\n",
    "    )\n",
    "    ),\n",
    "\n",
    "    ('ratio', CombineWithReferenceFeature(\n",
    "        variables_to_combine=['free sulfur dioxide'],\n",
    "        reference_variables=['total sulfur dioxide'],\n",
    "        operations=['div'],\n",
    "        new_variables_names=['percentage_free_sulfur']\n",
    "    )\n",
    "    ),\n",
    "\n",
    "    ('acidity', CombineWithReferenceFeature(\n",
    "        variables_to_combine=['fixed acidity'],\n",
    "        reference_variables=['volatile acidity'],\n",
    "        operations=['div', 'add'],\n",
    "        new_variables_names=['ratio_fixed_to_volatile', 'total_acidity']\n",
    "    )\n",
    "    ),\n",
    "\n",
    "    # scale features\n",
    "    ('scaler', StandardScaler()),\n",
    "\n",
    "    # Logistic Regression\n",
    "    ('LogisticRegression', LogisticRegression())\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('subtraction',\n",
       "                 CombineWithReferenceFeature(new_variables_names=['non_free_sulfur_dioxide'],\n",
       "                                             reference_variables=['free sulfur '\n",
       "                                                                  'dioxide'],\n",
       "                                             variables_to_combine=['total '\n",
       "                                                                   'sulfur '\n",
       "                                                                   'dioxide'])),\n",
       "                ('ratio',\n",
       "                 CombineWithReferenceFeature(new_variables_names=['percentage_free_sulfur'],\n",
       "                                             operations=['div'],\n",
       "                                             reference_variables=['total '\n",
       "                                                                  'sulfur '\n",
       "                                                                  'dioxide'],\n",
       "                                             variables_to_combine=['free '\n",
       "                                                                   'sulfur '\n",
       "                                                                   'dioxide'])),\n",
       "                ('acidity',\n",
       "                 CombineWithReferenceFeature(new_variables_names=['ratio_fixed_to_volatile',\n",
       "                                                                  'total_acidity'],\n",
       "                                             operations=['div', 'add'],\n",
       "                                             reference_variables=['volatile '\n",
       "                                                                  'acidity'],\n",
       "                                             variables_to_combine=['fixed '\n",
       "                                                                   'acidity'])),\n",
       "                ('scaler', StandardScaler()),\n",
       "                ('LogisticRegression', LogisticRegression())])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value_pipe.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_train = value_pipe.predict(X_train)\n",
    "pred_test = value_pipe.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Model train accuracy score: 0.7477414871438499\n",
      "\n",
      "Logistic Regression Model test accuracy score: 0.75\n"
     ]
    }
   ],
   "source": [
    "print('Logistic Regression Model train accuracy score: {}'.format(\n",
    "    accuracy_score(y_train, pred_train)))\n",
    "\n",
    "print()\n",
    "\n",
    "print('Logistic Regression Model test accuracy score: {}'.format(\n",
    "    accuracy_score(y_test, pred_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression Model test classification report: \n",
      "\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.73      0.73        74\n",
      "           1       0.77      0.77      0.77        86\n",
      "\n",
      "    accuracy                           0.75       160\n",
      "   macro avg       0.75      0.75      0.75       160\n",
      "weighted avg       0.75      0.75      0.75       160\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Logistic Regression Model test classification report: \\n\\n {}'.format(\n",
    "    classification_report(y_test, pred_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEYCAYAAABxx2wUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgsUlEQVR4nO3deZyd8/n/8ddbFiJKEpEEsdZOS9vYqoultZTYpaFIfVVo8S2toouq0lZpS1q6hCB8SSJ2WrFEUvzaZpMgElsRhIglISKRzMz1++P+DMc4M+dMzJxzn8z76XE/5tzb577mZFznc677c9+3IgIzM8ufVaodgJmZFecEbWaWU07QZmY55QRtZpZTTtBmZjnlBG1mllNO0Dkg6XlJIWmzaseSN5K2k3SbpFclLUnv1WhJ21U7thUhaS1JV0taIOltSddLWrvEPrunv49i0z0F2327mW1Oav/fzNpD52oH0NFJ2hXYOM0eCZxfvWjyJX1g/QeYDJwCLAA2B44APgvMrF50K+xGYAvgO0AD8FvgNuDLLezzCLBrk2UbAmOAu4tsvyewpGD+uRWM1arMCbr6jgQWkyWb3CRoSZ2AThGxrIphHAe8D+wXEe+nZQ8Af5Ok9j64pG4RsaT0lmW3tyuwN/DViHgwLZsLTJL0tYi4v9h+EfEO2QdVYVtfJkvwNxbZZUpEvNtWcVv1uMRRRSkJDgLuAK4Ctpa0fZHtviJpgqR309fiiZI+V7B+I0mjJL0h6T1Jj0k6Kq1r/Hq8XZM2J0q6qWD+GklTJR0s6QlgKbCzpHUlXSXpuVRieFrSBZK6Nmmvm6SLJM2R9H4qRfwmrbso7a8m+3xb0jJJ6zTzFvUAFhYk5w9Ek0tgJR0iaXKK8U1J/5C0UcH6PSVNkrRU0muS/ixpjYL1je/TPpLukPQucFlat2Eqq7yV3t97JG3ZTMwt2Q94rTE5p99jMvB8WtcaRwL/jIhXViAOqxFO0NW1B9AXGA3cBCwn+x/vA5J2B8andUOAbwIPAeun9X2AfwM7AmcAA4ERwAYrEM/GwEXAb8gSxvNAb+At4AfAvsDFZD3bPxXEKOB24LvA5cA3gHPTvpB9+GwCfLXJ8Y4D7oyI15uJ5xFgU0nDJG3TXNCSjgFuAf5L9oF3HPA0sE5avy0wDngDOCzFdhTZe97UCOBR4EBghKRewMPAlsBJqf3uwP2SuhXEMFHSxOZiTLYCniyyfHZaVxZJWwCfA0Y1s8l/JdVJekrSieW2azkUEZ6qNJElgwVA1zR/F/ACoIJt/g1MLVzWpI3fkJVI1m1m/e5AANs1WT4RuKlg/pq03Q4lYu5MltyWFsS9T9r3wBb2exgYWTC/KdlX9ANKHGtMajuAN4HrgAEF26wCzAVuaaGd0cAzZCWbxmWDUpu7NnmfLmmy7/npuL0KlvUE3gZOLlg2Hhhf4r27D7ityPL/A/7Vir+bnwPLCmMq+Hf4GVkZZT9gZPqdTq/237qnFZvcg66SVCI4FLg1PqzzjgY2Ip0QktQd2JkssTV3V6s9gXER8WobhDU3ImY0iVOSTpM0S9ISsp789cCqZCeqGmN4KyLuaKHtEcBhBWWFbwOvkfVsi4qIuoj4JrA9cA4wjSyx/lvS/mmzLYH1gKtbOPZOZO9zfcGym4E64EtNtv17k/mvkSXWdyR1ltQZWJRiGVAQ614RsVcLMbSlwcC9EfFW4cKIuCciLoiIeyPi7ogYQlaj/pkk/79eg/yPVj37kdVY/yGph6QeZL3a9/mwzNETENBS8l27xPrWeK3IstOA3wG3AgeRJbuT07rVWhHDjWQ95kGpJDIEuDYi6koFFRGPpcSzN1lCfhW4oODYlDj+ujT53VKyfhPo1WTbpu9Bb7Ky0vIm0x60voy0AFiryPKeaV1J6RzF1jRf3mjqJrLfceMyt7cc8SiO6mlMwmOLrDtC0mlk/9M2kCWY5rxZYv3S9LNrk+U9yWqyhYr10o8gK4X8tHFBkXpwqRiIiMWSRpP1nOeQ9b5b6vU2184LksYC3ys4NiWO/yrQp3BBOkG7Nll9/SOHaDL/FtlJ3GKjaxaVE3OBJyk+nG4rsqF25RhMNoTu9jK3jyY/rYa4B10FqXQxkKwXtEeT6QdkJw73jIjFwCTg2BaGlY0H9pHUt5n1L6efWxccfwPKPynVjaxXX+hbRWLoJemAEm2NIEtQvwD+ExHFTph9IJ0ALWZzPuzpPkVWgx7SQlOTgENSUm50KFkH5eESMY8HtgWeiIipTaanSuzb1N1AP0kflFUkDSCrxxcbz1zMYLITq+UOozuc7IN4TmsCtZyodhG8I05kJ9kC2LnIui5k/0Ndlea/QnZCaBxZUtmHLMEdkNavQ5aEnyZLUnuSlSDOLGhzCtnFCoeQjWKYRtarbHqScGqReC4i64V/Lx372tTWByceycow44B3yEaS7EmWxP9WpL2Zad+hZbxPfwIeJBs98VWy0SFXpv1/WOT9vB44ANgf+D3pZCJZgl0G3JnaGEr27WRcQRu7U/xkam/gRbKTtUelOAaRjVY5smC7kicJ03b3pPfvUOBgsg+Yh5psMwJ4tsi+u6QYD26m7ZuBs8jKZweQnVAN4NRq/817WrGp6gF0xCkliqdbWP9nYCGwapr/akpU76XlEygYbUF2YnFMSjrvkQ0TG1ywfjOy+vbilBAOovgojmIJeg2yUsRbaboy/c//kWRG1tP+HdmHxftkQ/R+VaS9C1KMa5bxPu2Sjv1M2ucN4F+Fv1vBtoeSffAsJSt7/B3YqGD9XmQ96aXA/PQer1GwvmiCTusaT0K+ln63F8hGXmxbsM1EYGIZv1OP1NZCsg+0G4DeTba5BnihyL6XFv5dFFn/6/Tv+x5ZGWQacEy1/949rfik9A9rVhGSJgNPRcQx1Y7FLO98ktAqItVa9yS7oObkEpubGU7QVjlTyL6e/zgiplQ5FrOa4BKHmVlOeZidmVlO5bbEsfiCo921t49Z65cTqh2C5VDdsrmf+Pazy994ruyc06X3pu1+u1twD9rMLLdy24M2M6uohvrS21SYE7SZGUB9yft2VZwTtJkZENFQ7RA+xgnazAygwQnazCyf3IM2M8spnyQ0M8sp96DNzPIpPIrDzCynfJLQzCynXOIwM8spnyQ0M8sp96DNzHLKJwnNzHLKJwnNzPIpwjVoM7N8cg3azCynXOIwM8sp96DNzHKqfnm1I/gYJ2gzM3CJw8wst1ziMDPLKfegzcxyygnazCyfIocnCVepdgBmZrkQDeVPJUjqIekmSU9Kmi1pV0m9JN0n6Zn0s2epdpygzcwgK3GUO5U2DBgXEVsB2wOzgbOB8RGxOTA+zbfICdrMDNqsBy1pLeArwAiAiFgWEQuBg4CRabORwMGlQnKCNjODVvWgJQ2VNLVgGlrQ0ibA68DVkqZLulJSd6BvRLyatpkH9C0Vkk8SmplBq8ZBR8RwYHgzqzsDnwdOjYhJkobRpJwRESEpSh3HPWgzM4C6uvKnlr0MvBwRk9L8TWQJ+zVJ6wKkn/NLNeQEbWYGbVaDjoh5wEuStkyL9gJmAXcAQ9KyIcDtpUJyicPMDNr6QpVTgesldQWeA44j6xDfKOl4YA4wqFQjTtBmZtCm9+KIiBnAgCKr9mpNO07QZmbgS73NzHLLd7MzM8up0qMzKs4J2swMIEoOS644J2gzM3AN2swst5ygzcxyyicJzcxyqr6+2hF8jBO0mRm4xGFmlltO0GZmOeUatJlZPkWDx0GbmeWTSxxmZjnlURxmZjnlHrSZWU45QVs5up1yCSxbSjQ0QEM9S6/6+QfrOu+8H6t+/Vss/v1JsOTdKkZpldS//3pcc9Uw+vTtTURw5ZXX86fLRtCzZw9GXf8XNtpoA+bMeYnBR53EwoVvVzvc2uSbJVm5llz3q48lYK3Zi06bfoaGt9+oUlRWLXV1dfzozPOYPmMma6zRncmTxnH/+AcZcuwgHpjwMBddfDln/uhkzjrzZH78k19XO9zalMMedLs9NFbSVpLOkvTHNJ0laev2Ol5H0PXrR7N8/OhcftJb+5o3bz7TZ8wE4N13F/Pkk8+w/nr9GDhwH669biwA1143lgMP3LeaYda2hih/qpB26UFLOgs4EhgNTE6L+wOjJI2OiAvb47grj2C1o84GgrpHHqBu+gQ6bfF5YtECGua/WO3grMo22qg/O2y/HZMmT6dvn97MmzcfyJJ43z69qxxdDetAoziOB7aNiOWFCyX9AXgCKJqgJQ0FhgL88cCd+J8dN2+n8PJt6cjziUULYPU1We1bZ9Hw5it02e1Alt7w22qHZlXWvfvq3DjmCn5wxrksWvTxcxDhb1crLDpQiaMBWK/I8nXTuqIiYnhEDIiIAR01OQNZcgZ47x3qn5pGpw23ZpUe69DthF/T7ZRL0Jq96PadC1D3taobqFVU586dGTvmCkaNupXbbrsbgNfmv0G/fn0A6NevD/Nff7OaIda2jlLiAE4Dxkt6BngpLdsQ2Aw4pZ2OuXLosipIsGwpdFmVTptsx/KHbuO9S07+YJNup1zCkhHneBRHB3PF8N8z+8lnuXTY8A+W3XXnvRx7zBFcdPHlHHvMEdx55z1VjLDGdZR7cUTEOElbADsB66fFc4EpEZG/Qk+OqPuarHrEadnrVTpRN/Nf1D/3WHWDsqrb7Ys7cszRh/PY47OYOuVeAM4550J+e/HljL7hrxz37SN58cWXGXzUSVWOtIbl8F4cymvNavEFR+czMKuqtX45odohWA7VLZurT9rG4p8PLjvndP/l6E98vHJ4HLSZGXScEoeZWc3JYYnDCdrMjHwOs3OCNjMD96DNzHLLCdrMLKc60KXeZmY1xc8kNDPLKydoM7Oc8igOM7Occg/azCynnKDNzPIp6l3iMDPLJ/egzczyqS2H2Ul6AVgE1AN1ETFAUi9gDLAx8AIwKCIWtNROuz001sysprT9E1X2iIgdImJAmj8bGB8RmwPj03yLnKDNzCB7GF+504o5CBiZXo8EDi61gxO0mRkQdQ1lT5KGSppaMA1t2hxwr6RpBev6RsSr6fU8oG+pmFyDNjODVvWMI2I4MLyFTb4UEXMl9QHuk/Rkk/1DUslaScketKSLJK0pqYuk8ZJel3R0yd/AzKyGREOUPZVsK2Ju+jkfuJXs+ayvSVoXIP2cX6qdckoce0fEO8ABZGceNwN+VMZ+Zma1o41q0JK6S/pU42tgb2AmcAcwJG02BLi9VEjllDgat9kfGBsRb0sVeV6imVnFtOEwu77ArSlPdgZuiIhxkqYAN0o6HpgDDCrVUDkJ+q5UP1kCfFfSOsDSFQ7dzCyP2uhCwoh4Dti+yPI3gb1a01bJBB0RZ0u6CHg7IuolvUc2XMTMbKURddWO4OPKOUm4OvA94C9p0XrAgOb3MDOrPdFQ/lQp5ZwkvBpYBnwxzc8FLmi3iMzMqqH9L1RptXIS9Kcj4iJgOUBEvAf4LKGZrVTy2IMu5yThMkndyK6MQdKngffbNSozswqrZOItVzkJ+lxgHLCBpOuB3YBvt2dQZmaVFvX5KwyUM4rjPkmPALuQlTa+HxFvtHtkZmYVVJM9aElfSS8XpZ/bSCIiHmy/sMzMKisaarAHzUcv616N7JryacCe7RKRmVkV1GQPOiIGFs5L2gC4tL0CMjOrhoja7EE39TKwdVsHYmZWTTXZg5b0J9IQO7Jx0zsAj7RjTGZmFddQi6M4gKkFr+uAURHx/9opHjOzqqjJk4QRMbLUNmZmta6mErSkx/mwtPGRVWRPbPlsu0VlZlZh0Wa3g247LfWgD6hYFGZmVVZTPeiImFPJQMzMqimPw+zKuR/0LpKmSHpX0jJJ9ZLeqURwZmaVUl+vsqdKKWcUx2XAYGAs2Y36jwW2aM+gzMwqrSZ70AAR8SzQKSLqI+JqYN/2DcvMrLKiQWVPlVJOD/o9SV2BGenZhK9SZmI3M6sVeRzF0WyilbRjenlM2u4UYDGwAXBY+4dmZlY5tdaDHi5pDWA02dWDs4DzKhOWmVll1TfkrzDQbEQR8TmysdB1wE2SHpV0tqSNKxWcmVmlRJQ/VUqLHxkR8VREnBcR25CN3lgLGC/J9+Iws5VKQ6jsqVLKut2opFWAPkBfoDswvz2DMjOrtDwOs2sxQUv6MnAkcDDwOFk9+vSIeLv9QzMzq5w8juJo6WZJLwFzyJLyLyKior3mtX45oZKHsxqx5JWHqh2CraQqWbooV0s96C/5fhxm1lHkcRSHb5ZkZkbxeytX24o8k9DMbKVTayUOM7MOo6ZGcTR5WOzHRMT/tktEZmZVkMOHerfYg57awjozs5VKUEM9aD8s1sw6krpaKnE0krQOcBawDbBa4/KI2LMd4zIzq6g89qDLGfh3PTAb2ITsbnYvAFPaMSYzs4praMVUKeUk6LUjYgSwPCL+GRH/A7j3bGYrlUBlT5VSToJenn6+Kml/SZ8DerVjTGZmFdfWPWhJnSRNl3RXmt9E0iRJz0oak55U1aJyEvQFktYCfgicAVwJnF5mjGZmNaEelT2V6ftk5eFGvwUuiYjNgAXA8aUaKJmgI+KuiHg7ImZGxB4R8YWIuKPcCM3MakGDyp9KkdQf2J+sQ4skkZWGb0qbjCS7S2iLyhnFcTVFLlhJtWgzs5VCQytqy5KGAkMLFg2PiOEF85cCZwKfSvNrAwsjoi7NvwysX+o45VzqfVfB69WAQ4BXytjPzKxmtOZmSSkZDy+2TtIBwPyImCZp908SU8kEHRE3Nzn4KODhT3JQM7O8acPhc7sBB0r6Blmndk1gGNBDUufUi+4PzC3V0IrcAHVzssdfmZmtNBqksqeWRMSPI6J/RGwMDAYeiIhvAROAw9NmQ4DbS8VUTg16ER/t/c8ju7LQzGylUd/+hzgLGC3pAmA6MKLUDuWUOD5Vahszs1pXzuiM1oqIicDE9Po5YKfW7F+yxCFpfDnLzMxqWQMqe6qUlu4HvRqwOtBbUk/4IKo1KWN4iJlZLam1R16dCJwGrAdM48ME/Q5wWfuGZWZWWe1R4vikWrof9DBgmKRTI+JPFYzJzKzi8vhElXKG2TVI6tE4I6mnpO+1X0hmZpVXr/KnSiknQZ8QEQsbZyJiAXBCu0VkZlYFebwfdDmXeneSpIgIyG6hB5S8TZ6ZWS3JY4mjnAQ9Dhgj6W9p/sS0zMxspZHDRxKWlaDPIrtr03fT/H3AFe0WkZlZFeSxB13O/aAbIuKvEXF4RBwOzAI8qsPMVir1rZgqpZweNOkxV0cCg4DngVvaMygzs0qrqXHQkrYgS8pHAm8AYwBFxB4Vis3MrGLyWOJoqQf9JPAQcEBEPAsgyc8iNLOVUh4TdEs16EOBV4EJkq6QtBdU8C4hZmYVFK2YKqXZBB0Rt0XEYGArshtNnwb0kfQXSXtXKD4zs4poy4fGtpVyRnEsjogbImIg2WNapuMb9pvZSqZmR3E0Spd5N/uwRDOzWtWQwxuOtipBm5mtrPJ4ktAJ2syM2rthv5lZh+EetJlZTtUpf31oJ2gzM1ziMDPLLZc4zMxyysPszMxyKn/p2QnazAxwicPMLLfqc9iHdoI2M8M9aDOz3Ar3oM3M8imPPeiStxu1yurffz3uv3csjz06gUdnPMCppxwPQM+ePRj3j1HMfuJhxv1jFD16rFXlSK2S3ln0Lqf/9AIGHnkCA48ayoyZswG4fuztDDzyBA761on8/vIRVY6ytjUQZU+V4h50ztTV1fGjM89j+oyZrLFGdyZPGsf94x9kyLGDeGDCw1x08eWc+aOTOevMk/nxT35d7XCtQi689K/stvMALvnVz1i+fDlLlr7P5GmPMuHh/3DzyMvp2rUrby5YWO0wa1r+ChzuQefOvHnzmT5jJgDvvruYJ598hvXX68fAgftw7XVjAbj2urEceOC+1QzTKmjRu4uZ9uhMDhu4DwBdunRhzU+twZjb/s7xRw+ia9euAKzds0cVo6x9dUTZU6W4B51jG23Unx22345Jk6fTt09v5s2bD2RJvG+f3lWOzipl7ivz6NljLX72qz/w1LPPsc2Wm3P2aSfxwotzmfboTP44fCSrdu3CD0/5Dp/Zestqh1uz8niSsOI9aEnHtbBuqKSpkqY2NCyuZFi507376tw45gp+cMa5LFr07sfWR+Tvj8naR119PbOffpZvHrI/N11zOd26rcaI626kvr6ed95ZxA3DL+GHJ3+HM875jf8uPoGGVkyVUo0Sx3nNrYiI4RExICIGrLJK90rGlCudO3dm7JgrGDXqVm677W4AXpv/Bv369QGgX78+zH/9zWqGaBXUr09v+q7Tm89uuxUAe+/+JWY9/Sx9+/Tma1/dDUl8ZpstkcSChW9XOdraFa34r1LaJUFLeqyZ6XGgb3scc2VyxfDfM/vJZ7l02IePfrzrzns59pgjADj2mCO48857qhWeVVjvtXvRr886PD/nZQD+M20Gn954Q/b88q5MfuRRAF548WWW19XR06N7Vlgee9Bqj69Ekl4D9gEWNF0F/Csi1ivVRueu63fI72q7fXFH/jnxNh57fBYNDdlbcM45FzJp8nRG3/BXNthgfV588WUGH3USCzrgWfslrzxU7RCq4smn/8vPLxzG8rrlbLDeupz/k9NZvdtq/OzXl/DUM8/RpUtnzjjlO+z8hR2qHWpVdOm9qT5pG0dvdGjZOef/5tzyiY9XjvZK0COAqyPi4SLrboiIo0q10VETtLWsoyZoa1lbJOijNjqk7Jxzw5xbmz2epNWAB4FVyQZi3BQR50raBBgNrA1MA46JiGUtHaddShwRcXyx5JzWlUzOZmaV1oY16PeBPSNie2AHYF9JuwC/BS6JiM3IqgvHl2rI46DNzGi7GnRkGodedUlTAHsCN6XlI4GDS8XkBG1mRtte6i2pk6QZwHzgPuC/wMKIqEubvAysX6odJ2gzM1pX4ii8ZiNNQz/SVkR9ROwA9Ad2ArZakZh8JaGZGVDfigETETEcGF7GdgslTQB2BXpI6px60f2BuaX2dw/azIy2K3FIWkdSj/S6G/B1YDYwATg8bTYEuL1UTO5Bm5nRphegrAuMlNSJrBN8Y0TcJWkWMFrSBcB0oOT9YZ2gzcxou5slRcRjwOeKLH+OrB5dNidoMzOo6I34y+UEbWZGPu8Q6QRtZgbUuwdtZpZPLnGYmeWUSxxmZjnlHrSZWU7l8ZmETtBmZrTuUu9KcYI2M8MlDjOz3HKCNjPLKY/iMDPLKfegzcxyyqM4zMxyqj7a8IajbcQJ2swM16DNzHLLNWgzs5xyDdrMLKcaXOIwM8sn96DNzHLKozjMzHLKJQ4zs5xyicPMLKfcgzYzyyn3oM3Mcqo+6qsdwsc4QZuZ4Uu9zcxyy5d6m5nllHvQZmY55VEcZmY55VEcZmY55Uu9zcxyyjVoM7Occg3azCyn3IM2M8spj4M2M8sp96DNzHLKozjMzHLKJwnNzHIqjyWOVaodgJlZHkQr/muJpA0kTZA0S9ITkr6flveSdJ+kZ9LPnqVicoI2MyPrQZc7lVAH/DAitgF2AU6WtA1wNjA+IjYHxqf5FjlBm5mR1aDLnVoSEa9GxCPp9SJgNrA+cBAwMm02Eji4VEzKY93FPkrS0IgYXu04LF/8d1E9koYCQwsWDS/2byFpY+BBYDvgxYjokZYLWNA43+xxnKDzT9LUiBhQ7TgsX/x3kW+S1gD+CfwqIm6RtLAwIUtaEBEt1qFd4jAza2OSugA3A9dHxC1p8WuS1k3r1wXml2rHCdrMrA2l8sUIYHZE/KFg1R3AkPR6CHB7qbY8Dro2uM5oxfjvIp92A44BHpc0Iy37CXAhcKOk44E5wKBSDbkGbWaWUy5xmJnllBO0mVlOOUHnnKR9JT0l6VlJJa88spWfpKskzZc0s9qxWPtygs4xSZ2Ay4H9gG2AI9Mlo9axXQPsW+0grP05QefbTsCzEfFcRCwDRpNdLmodWEQ8CLxV7Tis/TlB59v6wEsF8y+nZWbWAThBm5nllBN0vs0FNiiY75+WmVkH4ASdb1OAzSVtIqkrMJjsclEz6wCcoHMsIuqAU4B7yO4pe2NEPFHdqKzaJI0C/g1sKenldOmwrYR8qbeZWU65B21mllNO0GZmOeUEbWaWU07QZmY55QRtZpZTTtD2EZLqJc2QNFPSWEmrf4K2rpF0eHp9ZUs3epK0u6QvrsAxXpDUu8myqyWd2GTZwZLuLidWs7xwgramlkTEDhGxHbAMOKlwpaQVekxaRHwnIma1sMnuQKsTdDNGkV3UU2hwWm5WM5ygrSUPAZul3u1Dku4AZknqJOliSVMkPdbYW1XmsnT/6vuBPo0NSZooaUB6va+kRyQ9Kmm8pI3JPghOT733L0taR9LN6RhTJO2W9l1b0r2SnpB0JaAicY8Htip4gnJ34GvAbZJ+ntqbKWl4esDnRxT2yiUNkDSxsZ10L+bJkqZLOigt3zYtm5Hej83b4s03c4K2olJPeT/g8bTo88D3I2IL4Hjg7YjYEdgROEHSJsAhwJZk964+liI9YknrAFcAh0XE9sAREfEC8FfgktR7fwgYluZ3BA4DrkxNnAs8HBHbArcCGzY9RkTUkz3yvvGhnAOBiRHxDnBZROyYviF0Aw5oxdvyU+CBiNgJ2AO4OCX/k4BhEbEDMIDsroNmn5if6m1NdSt4EvFDZI+P/yIwOSKeT8v3Bj5bULNdC9gc+AowKiXIVyQ9UKT9XYAHG9uKiObua/w1YJuCDu6aktZIxzg07ft3SQua2X8U8DuyRD8YuC4t30PSmcDqQC/gCeDOZtpoam/gQElnpPnVyD4g/g38VFJ/4JaIeKbM9sxa5ARtTS1JPcEPpCS5uHARcGpE3NNku2+0YRyrALtExNIisZTjX8C6krYn+4AZLGk14M/AgIh4SdIvyJJsU3V8+O2ycL3Iev5PNdl+tqRJwP7APySdGBHFPpzMWsUlDlsR9wDfldQFQNIW6av+g8A3U416XbIyQFP/Ab6SSiJI6pWWLwI+VbDdvcCpjTOSdkgvHwSOSsv2A3oWCzCym8yMAUYCd6dE35hs30i98eZGbbwAfCG9PqzJ731qY91a0ufSz02B5yLij8DtwGebadesVZygbUVcCcwCHkkPLv0b2bexW4Fn0rpryb76f0REvA4MBW6R9ChZEoWszHBI40lC4H+BAemk2yw+HE1yHlmCf4Ks1PFiC3GOArZPP4mIhWT175lkyXZKM/udBwyTNBWoL1h+PtAFeCwd//y0fBAwM5WGtku/u9kn5rvZmZnllHvQZmY55QRtZpZTTtBmZjnlBG1mllNO0GZmOeUEbWaWU07QZmY59f8B0iDtcFf3u/gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "score = round(accuracy_score(y_test, pred_test), 3)\n",
    "cm = confusion_matrix(y_test, pred_test)\n",
    "\n",
    "sns.heatmap(cm, annot=True, fmt=\".0f\")\n",
    "plt.xlabel('Predicted Values')\n",
    "plt.ylabel('Actual Values')\n",
    "plt.title('Accuracy Score: {0}'.format(score), size=15)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAFNCAYAAAAQOlZzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA+FElEQVR4nO3deZyVc//H8den0iItKKS0oKYmWphK6K7siRLdLdxkuS2l7BFFhEjWNmv9EkqWKIqsiRtR2tPc0iJJ4q60aJv5/P4419z3GLOcaebMdc7M+/l4nMec6zrXuc57rpn6zPd7Xdf3a+6OiIiIJJ5SYQcQERGRfaMiLiIikqBUxEVERBKUiriIiEiCUhEXERFJUCriIiIiCUpFXEREJEGpiIskEDNbbWZ/mNk2M/vZzMab2QFZtjnRzD4ys61mtsXM3jKz5CzbVDazx83sh2Bf3wfL1XL4XDOz68xsiZltN7MfzexVMzs2lt+viORORVwk8Zzr7gcAzYDmwO0ZL5hZa+A9YCpwOFAPWAj8y8yODLYpC3wINAbOAioDrYHfgJY5fOYTwPXAdcBBQAPgTaBjfsObWZn8vkdEsmcasU0kcZjZauCf7v5BsPwQ0NjdOwbLnwKL3b1Plve9A2x090vM7J/A/cBR7r4tis+sDywHWrv7VzlsMwt40d2fC5YvDXKeHCw70Be4ASgDvAtsd/dbMu1jKvCJuz9qZocDI4G/AduAx9x9RN5HSKRkUUtcJEGZWS2gA7AiWN4fOBF4NZvNXwFOD56fBrwbTQEPnAr8mFMBz4fzgFZAMjAJ6G5mBmBmBwJnAC+bWSngLSI9CDWDz7/BzM4s4OeLFDsq4iKJ500z2wqsBX4BBgfrDyLyb3p9Nu9ZD2Sc7z44h21ykt/tc/KAu//H3f8APgUcaBO81hX4wt1/AloA1d19iLvvdveVwLNAj0LIIFKsqIiLJJ7z3L0S0A5oyP+K8yYgHaiRzXtqAL8Gz3/LYZuc5Hf7nKzNeOKR83gvAz2DVRcCLwXP6wCHm9nmjAdwB3BoIWQQKVZUxEUSlLt/AowHHg6WtwNfAH/PZvNuRC5mA/gAONPMKkb5UR8CtcwsJZdttgP7Z1o+LLvIWZYnAV3NrA6RbvbXg/VrgVXuXjXTo5K7nx1lXpESQ0VcJLE9DpxuZk2D5QFAr+B2sEpmdqCZ3Ufk6vN7gm1eIFIoXzezhmZWyswONrM7zOwvhdLdvwPGAJPMrJ2ZlTWz8mbWw8wGBJstAM43s/3N7GjgiryCu/t8Ir0DzwEz3X1z8NJXwFYzu83MKphZaTM7xsxa5PvoiBRzKuIiCczdNwITgLuC5c+AM4HziZzHXkPkNrSTg2KMu+8icnHbcuB94HcihbMaMCeHj7oOGAWMBjYD3wNdiFyABvAYsBvYADzP/7rG8zIxyDIx0/eUBpxD5Ba6Vfyv0FeJcp8iJYZuMRMREUlQaomLiIgkKBVxERGRBKUiLiIikqBUxEVERBKUiriIiEiCSrjZhKpVq+Z169YNO4aIiEiRmDdv3q/uXj271xKuiNetW5e5c+eGHUNERKRImNmanF5Td7qIiEiCUhEXERFJUCriIiIiCUpFXEREJEGpiIuIiCQoFXEREZEEpSIuIiKSoGJWxM1snJn9YmZLcnjdzGyEma0ws0VmdlyssoiIiBRHsWyJjwfOyuX1DkD94HEV8GQMs4iIiBQ7MRuxzd1nm1ndXDbpDExwdwe+NLOqZlbD3dfHKpOIiEhhmTjnB6YuWPeX9cmHV2bwuY2LJEOY58RrAmszLf8YrPsLM7vKzOaa2dyNGzcWSTgREZHcTF2wjmXrfwcgPT2d1atXs3z58iLNkBAXtrn7M+6e4u4p1atnOwa8iIhIkUuuUZlLa2/mm4cv4asHL6Txhve546wGRfb5YU6Asg44ItNyrWCdiIhI3Mip23zpui2w+UfOvuZKkpKS+PDDDznllFOKNFuYLfFpwCXBVeonAFt0PlxEROJN5m7zzBocsj9bFn3I0KFDWbRoUZEXcIhhS9zMJgHtgGpm9iMwGNgPwN2fAmYAZwMrgB3AZbHKIiIiUhDJNSoz+erWzJ49m3HjxjFu3DhKlSrFritbUa5cudByxfLq9J55vO7AtbH6fBERKZicupFLmmXrf6d+tfL06tWLCRMmUKdOHdauXUudOnVCLeCQIBe2iYhI0cupG7lkcar6Nj6b+ASTJk1i4MCBLFu2jDp16oQdDAj3wjYREYlzGd3IJdXOnTs55phjOLZOHUYvWkTDhg3DjvQnKuIiIsVUQbvDl63/neQalQsxUWLYtGkTw4cPZ+DAgVSsWJHPPvuMQw89FDMLO9pfqDtdRKSYKmh3eHKNynRulu0YXMWSuzNhwgSSkpIYNmwYH3/8MQCHHXZYXBZwUEtcRKRYK+nd4dFasmQJffr04dNPP6V169a89957NGvWLOxYeVIRFxEJWayuAi+p3eH74oYbbmDp0qU899xzXHbZZZQqlRgd1SriIiIhy+j2LuyCW9K6w/PD3ZkyZQqtW7fm8MMP57nnnuOAAw6gWrVqYUfLFxVxEZE4oG7vorNixQr69evHu+++y6233sqwYcOoW7du2LH2iYq4iCS04jAgibq9i8bOnTsZNmwYDzzwAGXLluWxxx6jb9++YccqkMTo9BcRyUFxGJBE3d5FY9CgQdx999106dKF5cuXc8MNN1CmTGK3ZRM7vYgI6oqWnK1du5Y//viDBg0a0L9/f8466yxOO+20sGMVGhVxEYk7+ekiV1e0ZGfPnj088cQT3H333bRq1YoPP/yQQw89lEMPPTTsaIVK3ekiEnfy00WurmjJ6tNPP6V58+b079+fU045hbFjx4YdKWbUEhdJUMXhgq6cZLSu1UUu+fXGG29w/vnnU6dOHaZOnUqnTp3CjhRTaomLJKjicEFXTtS6lvxIS0tj9erVAHTo0IEHHniApUuXFvsCDmqJiyQ0tValpJs3bx69e/dm48aNLFu2jAoVKjBgwICwYxUZtcRFRCThbNq0iWuvvZYWLVqwdu1a7r//fsqXLx92rCKnlriIiCSUlStX0rp1a3799Vf69evHkCFDqFKlStixQqEiLiIiCWHr1q1UqlSJunXr0q1bNy6//HKaN28edqxQqYiLxLmcrkLX/dFSUmzbto0hQ4Ywbtw4Fi9eTI0aNRg5cmTYseKCzomLxLmcrkLXFdxS3GXMNNaoUSOGDx9O586dKVu2bNix4opa4iIJQFehS0mze/duzjvvPN555x2aNGnC5MmTOfHEE8OOFXdUxEXiSHZd5+o2l5IkPT2dUqVKUbZsWerWrfvfmcYSfaKSWFF3ukgcya7rXN3mUlLMnDmTxo0bs2jRIgDGjBlTLGYaiyUdGZE4o65zKWl+/PFHbrzxRl577TUaNGjAjh07wo6UMFTERWJMM3KJ5GzkyJHcfvvtpKWlcd9993HLLbdQrly5sGMlDBVxkRjL6CKPpjir61xKmo0bN9KuXTtGjhxJvXr1wo6TcFTERYqAushFIjZu3Mitt95K165d6dixI4MHD6ZUqVKYWdjREpIubBMRkZhLS0vj6aefJikpiRdffJHvvvsOgNKlS6uAF4CKuIiIxNQ333zDiSeeyDXXXEPTpk1ZuHAhN9xwQ9ixigV1p4uISEwtWLCANWvW8NJLL9GzZ0+1vAuRiriIiBQqd+ell14iLS2NXr16cemll9K1a1cqV9adF4VN3ekiIlJoli1bRvv27bn44ouZOHEi7k6pUqVUwGNERVxERAps27Zt3HbbbTRt2pRFixbxzDPP8M4776jrPMbUnS6Si/wM1JITDeAiJcHcuXN56KGHuPzyy3nwwQepXr162JFKBLXERXKR0zSg+aEBXKS4WrlyJePHjwegXbt2pKamMnbsWBXwIpRnS9wifSEXAUe6+xAzqw0c5u5fxTydSBzQQC0if7Zr1y4eeughhg4dSoUKFejSpQtVqlShQYMGYUcrcaJpiY8BWgM9g+WtwOiYJRIRkbj13nvvceyxx3LXXXfRqVMnFi9eTJUqVcKOVWJFc068lbsfZ2bzAdx9k5mVjXEuERGJMz/99BPnnHMOderUYebMmZxxxhlhRyrxommJ7zGz0oADmFl1ID2mqUREJC7s2bOHqVOnAnD44Yczc+ZMFi9erAIeJ6Ip4iOAN4BDzOx+4DNgaDQ7N7OzzCzVzFaY2YBsXq9tZh+b2XwzW2RmZ+crvYiIxMxnn33G8ccfz3nnncecOXMAaN++PeXLlw85mWTItYibWSlgFXAr8ACwHjjP3V/Na8dB63000AFIBnqaWXKWzQYBr7h7c6AHkfPvIiISoo0bN3L55ZfTpk0bNm/ezBtvvEHLli3DjiXZyPWcuLunm9nooMguz+e+WwIr3H0lgJm9DHQGlmX+CCDjBtoqwE/5/AyRfMvPvd+6x1tKmrS0NE466SRWrVrFgAEDGDRoEBUrVgw7luQgmgvbPjSzC4Ap7u752HdNYG2m5R+BVlm2uRt4z8z6ARWB0/Kxf5F9knHvdzTFWfd4S0mxePFikpOTKV26NI8//jh169YlOTlr56nEm2iK+NXATUCame0M1rm7F0bzpCcw3t0fMbPWwAtmdoy7/+nCOTO7CrgKoHbt2oXwsVLS6d5vkYjNmzdz5513MmbMGJ588kmuuuoqzj5blyclijwvbHP3Su5eyt33C55XirKArwOOyLRcK1iX2RXAK8HnfAGUB6plk+EZd09x9xSNBCQiUnDuzosvvkjDhg0ZM2YMffr0oVu3bmHHknyKaux0M+sE/C1YnOXub0fxtq+B+mZWj0jx7gFcmGWbH4BTgfFm1ohIEd8YTSYREdl3V155JWPHjqVly5bMmDGD4447LuxIsg+iGXb1QaAF8FKw6nozO8ndb8/tfe6+18z6AjOB0sA4d19qZkOAue4+DbgZeNbMbiRykdul+TzvLiIiUdq+fTsAFStW5KKLLqJly5b885//pFQpTaORqCyvmmlmi4BmGeepg1vH5rt7kyLI9xcpKSk+d+7cMD5a4ly0V51nXNSmc+JSUrg7U6dO5frrr6d79+489NBDYUeSfDCzee6ekt1r0f75VTXTcw2SK3Ep2hnHdMW5lCQrV67k3HPPpUuXLlSuXJlOnTqFHUkKUTTnxB8A5pvZx4AROTf+l9HXROKBWtgi/zNp0iQuv/xyypQpwyOPPEK/fv3Yb7/9wo4lhSjPIu7uk8xsFpHz4gC3ufvPMU0lxVp+BlvJDw3MIhKxe/duypYtS7NmzTjvvPMYPnw4tWrVCjuWxECe3elm1gXY4e7TgovRdprZeTFPJsVWtN3e+aVucinp1q1bR/fu3fnHP/4BQKNGjZg0aZIKeDEWTXf6YHd/I2PB3Teb2WDgzZilkmJP3d4ihWfv3r2MHDmSu+66i71793LHHXeQnp6uq85LgGiKeHa/BVHdXy4iIrH17bff0qNHDxYtWsTZZ5/NyJEjOfLII8OOJUUkmmI818weJTIjGUBfYF7sIomISLSqVatGqVKlmDJlCueddx5mFnYkKULR9LX0A3YDk4PHTuDaWIYSEZHspaen8+yzz9KhQwfS0tKoXr0633zzDV26dFEBL4GiuTp9O8EtZWZ2ILBZo6qJiBS9+fPn07t3b+bMmcPf/vY3Nm3aRLVq1VS8S7AcW+JmdpeZNQyelzOzj4AVwAYz05ShIiJFZNu2bVx33XWkpKSwatUqJkyYwKxZs6hW7S/zRUkJk1t3encgNXjeK9j2EKAtMDTGuUREJFCmTBnee+89evfuTWpqKhdffLFa3wLkXsR3Z+o2PxOY5O5p7v4tujpdRCSmvv32Wy666CK2b99O+fLlmT9/PqNGjaJq1aphR5M4klsR32Vmx5hZdaA98F6m1/aPbSwRkZJp+/bt3H777TRt2pQZM2awePFiACpUqBByMolHuRXx64HXgOXAY+6+CsDMzgbmF0E2EZESI2OmseTkZB588EEuuugiUlNTOeGEE8KOJnEsx25xd58DNMxm/QxgRixDiYiURI899hiVK1dm9uzZtGnTJuw4kgB0bltEJCS7du3i0Ucf5ZJLLqFmzZpMnjyZgw46SDONSdQ0sK6ISAg++OADmjRpwh133MGrr74KwKGHHqoCLvmiIi4iUoR++uknevTowemnn056ejrvvvsuN9xwQ9ixJEHl2J1uZufn9kZ3n1L4cUREirchQ4bw5ptvcs8993DrrbdSvnz5sCNJAsvtnPi5ubzmgIq4iEgUPv/8cypVqsSxxx7LvffeS//+/TnqqKPCjiXFQG5Xp19WlEFERIqbX3/9lQEDBjB27Fi6dOnClClTqF69OtWrVw87mhQTuXWn35TbG9390cKPIyKS+NLT0xk3bhy33XYbv//+O/379+euu+4KO5YUQ7l1p1cqshQiIsXIs88+yzXXXEObNm0YM2YMxxxzTNiRpJjKrTv9nqIMIiKSyLZs2cKqVato1qwZvXr1omrVqnTr1k0TlUhM5TnYi5mVB64AGgP/vYzS3S+PYS4RkYTg7rz88svcdNNNVKhQgX//+9+UL1+e7t27hx1NSoBo7hN/ATiMyExmnwC1gK2xDCUikgiWL1/OaaedxoUXXkitWrV45ZVXKFNGA2FK0Ynmt+1od/+7mXV29+fNbCLwaayDiYjEswULFtCyZUsqVqzIk08+yZVXXknp0qXDjiUlTDQt8T3B181mdgxQBTgkdpFEROLXmjVrAGjatCl33303qampXHPNNSrgEopoWuLPmNmBwCBgGnAAoHslSqiJc35g6oJ1BdrHsvW/k1yjciElEikaq1ev5rrrrmPWrFmkpqZSo0YN7rjjjrBjSQmXZxF39+eCp7OBI2MbR+Ld1AXrClyEk2tUpnOzmoWYSiR2du3axSOPPMJ9991HqVKluPvuu6lWrVrYsUSA6K5OHwo85O6bg+UDgZvdfVCMs0mcSq5RmclXtw47hkjMbdu2jRYtWrB8+XIuuOACHnvsMY444oiwY4n8VzTnxDtkFHAAd98EnB2zRCIiIdu+fTsABxxwAOeffz4zZszgtddeUwGXuBNNES9tZuUyFsysAlAul+1FRBLS3r17eeKJJzjiiCNYsGABAPfffz8dOnQIN5hIDqK5sO0l4EMz+79g+TLg+dhFEhEpel988QW9e/dm4cKFnHnmmVSurIsvJf5Fc2HbMDNbCJwWrLrX3WfGNpaISNHp27cvo0ePpmbNmrz22mucf/75Gi5VEkK0Qwt9C+x19w/MbH8zq+TuGrVNRBKWu/+3UB922GHccsstDB48mAMOOCDkZCLRy/OcuJldCbwGPB2sqgm8GcNMIiIxtXDhQk466STefvttAAYNGsTw4cNVwCXhRHNh27XAScDvAO7+HRqxTUQS0O+//84NN9zAcccdx4oVK9i9e3fYkUQKJJoivsvd//ubbmZlAI9dJBGRwvfmm2/SsGFDRowYwdVXX01qairnn39+2LFECiSac+KfmNkdQAUzOx3oA7wV21giIoVr8+bNHH744UydOpUWLVqEHUekUETTEh8AbAQWA1cDM9x9YDQ7N7OzzCzVzFaY2YActulmZsvMbGkwQ5qISIHt2LGDgQMH8tRTTwFwySWXMGfOHBVwKVbyLOLunu7uz7r73929K7DGzN7P631mVhoYDXQAkoGeZpacZZv6wO3ASe7eGLhhH74HEZE/eeutt0hOTmbo0KEsXboUgFKlSmmmMSl2ciziZnaKmf3bzLaZ2YtmdqyZzQUeAJ6MYt8tgRXuvjI4p/4y0DnLNlcCo4OhXHH3X/bt2xARiUwT2rlzZzp16kTFihWZNWsWI0eODDuWSMzk1hJ/BLgKOJjILWZfAOPd/Xh3nxLFvmsCazMt/xisy6wB0MDM/mVmX5rZWdFHFxH5s1WrVvHhhx/y0EMPsWDBAtq2bRt2JJGYyu3CNnf3WcHzN81snbuPisHn1wfaAbWA2WZ2bOYJVwDM7Coif1BQu3btQo4gIonso48+YsGCBdx00020a9eOH374gYMOOijsWCJFIrciXtXMMt9/USbzchSt8XVA5il/agXrMvsRmOPue4BVZvZvIkX968wbufszwDMAKSkpur2tiEyc8wNTF/z5R1bQucRFCsv69eu55ZZbmDhxIklJSfTp04fy5curgEuJklt3+ifAuZkeszM9PyeKfX8N1DezemZWFugBTMuyzZtEWuGYWTUi3esro48vsTR1wTqWrf/9T+uSa1Smc7OsZ0VEis7evXsZMWIEDRs25PXXX2fw4MHMnz+f8uXLhx1NpMjl2BJ398sKsmN332tmfYGZQGlgnLsvNbMhwFx3nxa8doaZLQPSgP7u/ltBPlcKV3KNyky+unXYMUT+a82aNfTv35/27dszcuRI6tevH3YkkdBEOwHKPnH3GcCMLOvuyvTcgZuCh4hItn777TdeeeUVevfuzVFHHcX8+fNp1KiRZhqTEi+awV5EREKRnp7OuHHjSEpKol+/fqSmpgKQnJysAi6CiriIxKmFCxfSpk0brrjiCho1asT8+fNJSkoKO5ZIXImqO93MGrr78oyvsQ4lIiXbrl27OPPMM0lLS2P8+PFccsklanmLZCPalvjELF9FRAqVu/P222+TlpZGuXLleP3110lNTaVXr14q4CI5yG93uv4liUihS01N5fTTT+fcc8/l5ZdfBuCkk07SPd8iedA5cREJzY4dOxg0aBDHHnssc+fOZfTo0fTo0SPsWCIJI6a3mImI5KZr16688847XHzxxQwfPpxDDz007EgiCSW/LXENeSoiBbJmzRq2bt0KwKBBg5g1axYTJkxQARfZB9EWccvyVUQkX3bv3s2DDz5Io0aNuO+++wA48cQTNdOYSAFE253eJstXEZGoffzxx/Tp04fly5fTpUsXrr322rAjiRQLUbXE3X1b5q8iItEaPnw4p5xyCrt27eLtt99mypQpmlJYpJDowjYRKXR79+5l27ZtVK1alXPPPZetW7dy++23U6FChbCjiRQrusVMRArVnDlzaNmyJf/85z8BaNiwIUOGDFEBF4kBFXERKRS//fYbV199Na1bt2bDhg1069Yt7EgixZ6600WkwD777DO6dOnCpk2buOmmmxg8eDCVKlUKO5ZIsaciLiL7bO/evZQpU4aGDRtywgkncP/999OkSZOwY4mUGFEXcTPb3913xDKMxN7EOT8wdcG6qLZdtv53kmtUjnEiSURbt25l8ODBfPnll3z66adUq1aNt956K+xYIiVOnufEzexEM1sGLA+Wm5rZmJgnk5iYumAdy9b/HtW2yTUq07lZzRgnkkTi7rzyyis0bNiQxx9/nGOPPZZdu3aFHUukxIqmJf4YcCYwDcDdF5rZ32KaSmIquUZlJl/dOuwYkmA2bNjAxRdfzPvvv0/z5s2ZMmUKrVq1CjuWSIkW7WAva7OsSotBFhGJY1WrVmXLli2MGjWKr7/+WgVcJA5EU8TXmtmJgJvZfmZ2C/BtjHOJSByYPn067dq1Y9u2bZQrV44vv/ySa6+9ltKlS4cdTUSIrohfA1wL1ATWAc2APjHMJCIh++GHH+jSpQvnnHMOv/zyC+vWRS6GNNMcSCLxJJoinuTuF7n7oe5+iLv/A2gU62AiUvTS0tIYNmwYjRo14r333uPBBx9kwYIFJCUlhR1NRLIRTREfGeU6EUlwpUqV4t133+WMM85g2bJl3HbbbZQtWzbsWCKSgxyvTjez1sCJQHUzuynTS5UBnRATKSZ+/vlnBg4cyD333EOtWrV4++23qVixYtixRCQKubXEywIHECn0lTI9fge6xj6aiMRSWloao0aNIikpiRdffJEvvvgCQAVcJIHk2BJ390+AT8xsvLuvKcJMIhJjc+bMoXfv3syfP5/TTz+d0aNHU79+/bBjiUg+RTPYyw4zGw40BspnrHT3U2KWSkRi6plnnmHDhg288sordO3aVVediySoaIr4S8Bk4Bwit5v1AjbGMpQUXE5jpGs89JIpPT2dCRMm0LRpU5o3b87DDz/MY489RuXK+l0QSWTRXJ1+sLuPBfa4+yfufjmgVnicy2mMdI2HXvIsXryYtm3bctlll/Hss88CcOCBB6qAixQD0bTE9wRf15tZR+An4KDYRZLCojHSS7atW7dy991388QTT1C1alXGjh3LpZdeGnYsESlE0RTx+8ysCnAzkfvDKwM3xDKUiBTc008/zaOPPspVV13F0KFDOfjgg8OOJCKFLM8i7u5vB0+3AO0BzOykWIYSkX3z3XffsWHDBk4++WT69u1L27ZtadGiRdixRCRGcjwnbmalzaynmd1iZscE684xs8+BUUWWUETy9Mcff3DXXXdxzDHH0Lt3b9yd8uXLq4CLFHO5tcTHAkcAXwEjzOwnIAUY4O5vFkE2EYnCjBkz6Nu3L6tWreKiiy5i+PDhumVMpITIrYinAE3cPd3MygM/A0e5+29FE01E8vLRRx/RsWNHGjZsyEcffUT79u3DjiQiRSi3Ir7b3dMB3H2nma1UAS86Od3nHS3dD1587dmzh4ULF5KSkkL79u15/vnn6dGjhyYqESmBcrtPvKGZLQoeizMtLzazRUUVsKTK6T7vaOl+8OLpk08+oVmzZrRv357ffvsNM+OSSy5RARcpoXJriWvO8JDpPm/JsGHDBvr3788LL7xA3bp1mThxom4ZE5FcJ0DRpCcicWDjxo00bNiQ7du3M3DgQO644w7233//sGOJSByIZrAXEQnBjz/+SK1atahevTp33nknHTt2JCkpKexYIhJHohk7fZ+Z2VlmlmpmK8xsQC7bXWBmbmYpscwjkgg2bdpE7969qVevHvPnzwfgpptuUgEXkb+IqoibWQUzy9f/IGZWGhgNdACSgZ5mlpzNdpWA64E5+dm/SHHj7owfP56kpCSeffZZ+vbty1FHHRV2LBGJY3kWcTM7F1gAvBssNzOzaVHsuyWwwt1Xuvtu4GWgczbb3QsMA3ZGG1qkuHF3zjzzTC677DKOPvpo5s2bp6lCRSRP0bTE7yZSkDcDuPsCoF4U76sJrM20/GOw7r/M7DjgCHefHsX+RIqdP/74AwAzo0OHDowdO5bPPvuMpk2bhpxMRBJBVFORuvuWLMM4ekE/2MxKAY8Cl0ax7VXAVQC1a9cu6EeHJj8DuGiwluLN3Xn99de54YYbGDNmDJ06deLGG28MO5aIJJhoWuJLzexCoLSZ1TezkcDnUbxvHZGx1zPUCtZlqAQcA8wys9XACcC07C5uc/dn3D3F3VOqV68exUfHp/wM4KLBWoqv7777jg4dOvD3v/+d6tWrc9hhh4UdSUQSVDQt8X7AQGAXMBGYCdwXxfu+BuqbWT0ixbsHcGHGi+6+BaiWsWxms4Bb3H1utOETkQZwKdkee+wxBgwYQPny5XniiSfo06cPZcroTk8R2TfR/O/R0N0HEinkUXP3vWbWl0jRLw2Mc/elZjYEmOvu0Vwcl5By6jZXF3nJ5e6YGQcffDBdu3bl4YcfpkaNGmHHEpEEZ+65n942s4+Bw4DXgMnuvqQoguUkJSXF586N78Z696e/yLFgd25WkwtbJe55fcmftWvXcuONN9KmTRuuv/76sOOISAIys3nunu04Knm2xN29vZkdBnQDnjazykSKeTRd6iWWus1Ltj179vD4449zzz33kJ6eTps2bcKOJCLFUFQn49z9Z2BE0Cq/FbiL6M6LF3vZdZ2r27xkmzNnDldccQVLly6lU6dOPPHEE9StWzfsWCJSDOVZxM2sEdAduAD4DZgM3BzjXAkj44rzzEVbV5aXbDt37mT79u1MnTqVTp06hR1HRIqxaFri44gU7jPd/acY50lI6jov2dLS0nj66afZsGED99xzD23btuXf//43++23X9jRRKSYi+acuKqTSA6+/vprevfuzbx58zjjjDNIS0ujdOnSKuAiUiRyHOzFzF4Jvi42s0WZHovNbFHRRRSJP5s2baJPnz60atWKn376iUmTJvHuu+9SunTpsKOJSAmSW0s8436Yc4oiiEgi+fXXX3n++ee5/vrrueeeezRRiYiEIseWuLuvD572cfc1mR9An6KJJxI/lixZwuDBgwGoX78+a9as0UxjIhKqaMZOPz2bdR0KO4hIvNq2bRv9+/enefPmjBo1inXrIrcUVqtWLY93iojEVm7nxHub2WIgKcs58VWAzolLsZcx01ijRo14+OGH6dWrF6mpqdSsqdsHRSQ+5HZOfCLwDvAAMCDT+q3u/p+YphKJA1u3buWaa66hZs2aTJ48mRNPPDHsSCIif5Jbd7q7+2rgWmBrpgdmdlDso4kUvZ07dzJ69GjS0tKoXLkys2bNYu7cuSrgIhKX8mqJnwPMAxywTK85cGQMc4kUuXfffZe+ffvy/fffc+SRR9KhQwcaN24cdiwRkRzldnX6OcHXeu5+ZPA146ECLsXG2rVr6dq1Kx06dKBMmTJ88MEHdOigazdFJP7leXW6mZ1kZhWD5/8ws0fNTHNpSrHg7lxwwQVMnz6d+++/n4ULF3LqqaeGHUtEJCrRjJ3+JNDUzJoSmfjkOeAFoG0sg4nE0meffUbTpk2pVKkSTz/9NFWrVqVevXphxxIRyZdo7hPf6+4OdAZGuftooFJsY4nExi+//MKll15KmzZteOSRRwBo3ry5CriIJKRoWuJbzex24GKgjZmVAjS7gySUtLQ0nn32WW6//Xa2b9/OHXfcwa233hp2LBGRAommJd4d2AVc7u4/A7WA4TFNJVLIbr75Znr37s1xxx3HokWLuP/++9l///3DjiUiUiDRTEX6s5m9BLQws3OAr9x9QuyjiRTMpk2b2LVrF4cddhh9+vThhBNOoHv37phZ3m8WEUkA0Vyd3g34Cvg70A2YY2ZdYx1MZF+5Oy+88AINGzbk2muvBaBBgwb06NFDBVxEipVozokPBFq4+y8AZlYd+AB4LZbBRPbF0qVL6dOnD7Nnz6ZVq1YMGjQo7EgiIjETzTnxUhkFPPBblO8TKVJTpkyhWbNmLFmyhGeeeYbPP/+c5s2bhx1LRCRmommJv2tmM4FJwXJ3YEbsIolEz93ZsmULVatWpW3btvTu3Zu77rpL04SKSIkQzYVt/c3sfODkYNUz7v5GbGOJ5O3777+nX79+bNiwga+++oqDDz6YESNGhB1LRKTI5FjEzaw+8DBwFLAYuMXd1xVVMJGc7Ny5k2HDhvHAAw9QtmxZ7r333rAjiYiEIreW+DhgAjAbOBcYCZxfFKHi0cQ5PzB1wV//hlm2/neSa1QOIVHJtGLFCs466yy+//57evTowSOPPMLhhx8ediwRkVDkVsQrufuzwfNUM/umKALFq6kL1mVbsJNrVKZzs5ohpSo50tLSKF26NLVr16Zx48Y89dRTnHbaaWHHEhEJVW5FvLyZNed/84hXyLzs7iWuqCfXqMzkq1uHHaNE2bNnDyNGjOC5557jq6++olKlSkydOjXsWCIicSG3Ir4eeDTT8s+Zlh04JVahRCAy01jv3r1ZsmQJHTt2ZPv27VSqpLl3REQy5FjE3b19UQYRybBz50569+7N+PHjqV27Nm+++SadOnXSaGsiIllo0BaJO+XKlePnn39mwIABLFu2jM6dO6uAi4hkI5rBXoqtnK44z46uQo+tefPm0b9/f55//nmOOOIIpk+fTqlS+htTRCQ3Jfp/yYwrzqOhq9BjY/PmzfTt25cWLVqwbNkyVq5cCaACLiIShTxb4hbpx7wIONLdh5hZbeAwd/8q5umKgK44D8+LL77IzTffzK+//krfvn259957qVKlStixREQSRjTd6WOAdCJXow8BtgKvAy1imEtKgFmzZlGvXj3effddTVQiIrIPoinirdz9ODObD+Dum8ysbIxzSTG0fft27r33Xrp27UpKSgojRoygfPny6joXEdlH0RTxPWZWmsi94RnziafHNJUUK+7O1KlTue6661i7di0HHnggKSkp7L///mFHExFJaNE0gUYAbwCHmNn9wGfA0JimkmJj5cqVnHPOOXTp0oWqVavy2Wefcdttt4UdS0SkWIhmKtKXzGwecCqRIVfPc/dvY55MioVJkyYxe/ZsHn30Ufr27ct+++0XdiQRkWIjz5Z4cDX6DuAtYBqwPViXJzM7y8xSzWyFmQ3I5vWbzGyZmS0ysw/NrE5+vwGJP++99x4zZ84E4JZbbmH58uXceOONKuAiIoUsmu706cDbwdcPgZXAO3m9KTiPPhroACQDPc0sOctm84EUd28CvAY8FH10iTfr1q2jW7dunHnmmQwfPhyIjL5Ws6burxcRiYU8i7i7H+vuTYKv9YGWwBdR7LslsMLdV7r7buBloHOWfX/s7juCxS+BWvmLL/Fgz549PProozRs2JC33nqLe++9l+nTp4cdS0Sk2Mv3sKvu/o2ZtYpi05rA2kzLPwK5ve8KomjhS/yZPn06N998Mx07dmTEiBEceeSRYUcSESkRohmx7aZMi6WA44CfCjOEmf0DSAHa5vD6VcBVALVrR3U6XmJs48aNfPPNN5x55pl07tyZjz/+mLZt22qiEhGRIhTNOfFKmR7liJwb75zrOyLWAUdkWq4VrPsTMzsNGAh0cvdd2e3I3Z9x9xR3T6levXoUHy2xkp6eztNPP01SUhI9e/Zk+/btmBnt2rVTARcRKWK5tsSDi9Mqufst+7Dvr4H6ZlaPSPHuAVyYZf/NgaeBs9z9l334DClC8+bNo0+fPnz11Ve0bduWMWPGULFixbBjiYiUWDkWcTMr4+57zeykfdlx8N6+wEygNDDO3Zea2RBgrrtPA4YDBwCvBq24H9y90758nsTW6tWradWqFdWqVePFF1/kwgsvVMtbRCRkubXEvyJy/nuBmU0DXgW2Z7zo7lPy2rm7zwBmZFl3V6bnp+U3sBQdd2fevHmkpKRQt25dnn/+eTp27EjVqlXDjiYiIkR3Trw88BuRWczOAc4Nvkox9u2333LKKafQqlUrlixZAsBFF12kAi4iEkdya4kfElyZvoTI5CeZ+049pqkkNBkzjT3yyCNUqlSJMWPGkJycdYweERGJB7kV8dJEzldnd+JTRbwY2rNnD8cffzypqalcdtllDBs2DN0NICISv3Ir4uvdfUiRJZHQrF+/nsMOO4z99tuP/v37k5SUxMknnxx2LBERyUNu58R16XExt2vXLu69916OPPJIpk6dCsAVV1yhAi4ikiBya4mfWmQppMi9//77XHvttXz33Xd069aNFi1ahB1JRETyKceWuLv/pyiDSNHp168fZ5xxBgAzZ85k8uTJmmlMRCQB5XsCFElMe/fuBaBMmTKcdNJJHHLIIdx6662UK1cu5GQiIrKvorlPXBLcv/71L44//nhGjRoFQI8ePbjzzjtVwEVEEpyKeDH266+//vdCtf/85z+aIlREpJhRES+mpkyZQlJSEhMmTODWW2/l22+/pVMnDUsvIlKc6Jx4MePumBnVq1enSZMmjBo1isaNG4cdS0REYkBFvJjYsmULd955J2XKlOHRRx+lTZs2fPTRR5ppTESkGFN3eoJzdyZOnEhSUhKjRo0iLS0N98iouCrgIiLFm1riCez777/nyiuv5OOPP6ZFixZMnz6d448/PuxYIiJSRFTEE5i7s3z5cp566in++c9/Urp06bAjiYhIEVIRTzDTpk1j5syZjB49mqOPPprVq1dTtmzZsGOJiEgIdE48QaxatYpOnTrRuXNnPvnkEzZt2gSgAi4iUoKpiMe5Xbt2MXToUBo3bsxHH33Eww8/zPz58znwwAPDjiYiIiFTd3qc27FjB48//jgdO3bkscceo1atWmFHEhGROKGWeBz66aefGDBgAHv37uXAAw9k8eLFvPrqqyrgIiLyJyricWTv3r08/vjjNGzYkMcff5x58+YBcOihh4acTERE4pGKeJz4/PPPOf7447nxxhs56aSTWLp0Ka1atQo7loiIxDGdE48D6enpXHXVVWzZsoXXX3+dLl26aLQ1ERHJk4p4SNLT03n++ee54IILqFy5MlOmTOHwww/ngAMOCDuaiIgkCHWnh2D+/PmceOKJXH755YwfPx6ABg0aqICLiEi+qIgXoS1btnD99deTkpLCqlWrmDBhAv369Qs7loiIJCh1pxehPn36MGnSJPr06cN9991H1apVw44kIiIJTEU8xpYvX07FihU54ogjGDJkCDfeeCMpKSlhxxIRkWJA3ekxsmPHDgYOHEiTJk24/fbbATjqqKNUwEVEpNCoJR4Db731Fv369WPNmjX06tWLhx56KOxIIiJSDKklXsjGjBlDp06dOOCAA/jkk08YP348hxxySNixRESkGFJLvBDs2rWLDRs2ULt2bXr06MGuXbvo27cv++23X9jRRESkGFNLvIA++OADmjRpwnnnnUd6ejoHHXQQN954owq4iIjEnIr4Pvrpp5/o2bMnp59+OmlpaQwdOpRSpXQ4RUSk6Kg7fR/MnTuXU045hd27d3P33Xdz2223Ub58+bBjiYhICaMing+///47lStXpkmTJlx00UXcfPPNHH300WHHEhGREkr9v1H47bffuPLKK2ncuDFbt26lbNmyPPnkkyrgIiISKhXxXKSnpzN27FiSkpL4v//7P3r06KHz3iIiEjfUnZ6DzZs3c/bZZ/PFF19w8skn8+STT3LMMceEHUtEROS/VMSzSE9Pp1SpUlSpUoW6dety9dVXc8kll2BmYUcTERH5k5j2DZvZWWaWamYrzGxANq+XM7PJwetzzKxuLPPkxt15+eWXSUpKYu3atZgZEydOpFevXirgIiISl2JWxM2sNDAa6AAkAz3NLDnLZlcAm9z9aOAxYFis8uQmNTWV008/nZ49e1KlShW2bt0aRgwREZF8iWVLvCWwwt1Xuvtu4GWgc5ZtOgPPB89fA061Im72rl69mmOPPZa5c+cyevRo5syZQ3Jy1r81RERE4k8si3hNYG2m5R+Dddlu4+57gS3AwVl3ZGZXmdlcM5u7cePGQguYfHhlyv2xkR49epCamkqfPn0oXbp0oe1fREQklhLiwjZ3fwZ4BiAlJcULa7+Dz23MoLMbqnCLiEhCimVLfB1wRKblWsG6bLcxszJAFeC3GGb6CxVwERFJVLEs4l8D9c2snpmVBXoA07JsMw3oFTzvCnzk7oXW0hYRESnOYtad7u57zawvMBMoDYxz96VmNgSY6+7TgLHAC2a2AvgPkUIvIiIiUYjpOXF3nwHMyLLurkzPdwJ/j2UGERGR4koDgYuIiCQoFXEREZEEpSIuIiKSoFTERUREEpSKuIiISIJSERcREUlQKuIiIiIJyhJtgDQz2wisKcRdVgN+LcT9lVQ6jgWnY1hwOoYFp2NYcIV9DOu4e/XsXki4Il7YzGyuu6eEnSPR6TgWnI5hwekYFpyOYcEV5TFUd7qIiEiCUhEXERFJUCriwTzlUmA6jgWnY1hwOoYFp2NYcEV2DEv8OXEREZFEpZa4iIhIgioxRdzMzjKzVDNbYWYDsnm9nJlNDl6fY2Z1Q4gZ16I4hjeZ2TIzW2RmH5pZnTByxrO8jmGm7S4wMzczXSWcjWiOo5l1C34fl5rZxKLOGO+i+Pdc28w+NrP5wb/ps8PIGa/MbJyZ/WJmS3J43cxsRHB8F5nZcTEJ4u7F/gGUBr4HjgTKAguB5Czb9AGeCp73ACaHnTueHlEew/bA/sHz3jqG+T+GwXaVgNnAl0BK2Lnj7RHl72J9YD5wYLB8SNi54+kR5TF8BugdPE8GVoedO54ewN+A44AlObx+NvAOYMAJwJxY5CgpLfGWwAp3X+nuu4GXgc5ZtukMPB88fw041cysCDPGuzyPobt/7O47gsUvgVpFnDHeRfN7CHAvMAzYWZThEkg0x/FKYLS7bwJw91+KOGO8i+YYOlA5eF4F+KkI88U9d58N/CeXTToDEzziS6CqmdUo7BwlpYjXBNZmWv4xWJftNu6+F9gCHFwk6RJDNMcwsyuI/BUq/5PnMQy63I5w9+lFGSzBRPO72ABoYGb/MrMvzeysIkuXGKI5hncD/zCzH4EZQL+iiVZs5Pf/zH1SprB3KGJm/wBSgLZhZ0kkZlYKeBS4NOQoxUEZIl3q7Yj0CM02s2PdfXOYoRJMT2C8uz9iZq2BF8zsGHdPDzuY/E9JaYmvA47ItFwrWJftNmZWhkj30W9Fki4xRHMMMbPTgIFAJ3ffVUTZEkVex7AScAwwy8xWEzmPNk0Xt/1FNL+LPwLT3H2Pu68C/k2kqEtENMfwCuAVAHf/AihPZExwiU5U/2cWVEkp4l8D9c2snpmVJXLh2rQs20wDegXPuwIfeXB1ggBRHEMzaw48TaSA6xzkX+V6DN19i7tXc/e67l6XyHUFndx9bjhx41Y0/57fJNIKx8yqEeleX1mEGeNdNMfwB+BUADNrRKSIbyzSlIltGnBJcJX6CcAWd19f2B9SIrrT3X2vmfUFZhK5KnOcuy81syHAXHefBowl0l20gsjFCj3CSxx/ojyGw4EDgFeDawJ/cPdOoYWOM1EeQ8lDlMdxJnCGmS0D0oD+7q6etUCUx/Bm4Fkzu5HIRW6XqmHzP2Y2icgfitWC6wYGA/sBuPtTRK4jOBtYAewALotJDv1MREREElNJ6U4XEREpdlTERUREEpSKuIiISIJSERcREUlQKuIiIiIJSkVc4o6ZpZnZgkyPurlsu60QPm+8ma0KPuubYHSq/O7jOTNLDp7fkeW1zwuaMdhPxnFZYmZvmVnVPLZvFuuZp8zs78EsYen7MiiNmR1qZm+b2cJgxrEZhZwvxcxGBM/LmdkHwTHsnvlnlsN7O2XM7mVm5+W2bQ7vP8EiMyIuMLNvzezuAn0zItnQLWYSd8xsm7sfUNjb5rKP8cDb7v6amZ0BPOzuTQqwvwJnymu/ZvY88G93vz+X7S8lMgta38LOkukzGgHpRAb5uSW/A9OY2dPAMnd/Ilhu4u6LCj9ppKgC97n7afvw3vEEvyP5eE8q0M3dF5pZaSDJ3Zfl97Oz7LO0u6cVZB9SvKglLnHPzA6wyPzk35jZYjP7y8xfZlbDzGZnaqm2CdafYWZfBO991czyKq6zgaOD994U7GuJmd0QrKtoZtODluMSM+serJ8VtPoeBCoEOV4KXtsWfH3ZzDpmyjzezLqaWWkzG25mX1tk3uGrozgsXxBMpmBmLYPvcb6ZfW5mSRYZhWsI0D1Ty7OiReZA/irYNrsZ1PLF3b9199QC7KIGkSFSM/a3CMDM2gU/z+kWmfP6KYuMLZ/jz9TMWgTf/8Lge6wU7OdtMzsEeBFoERyPozJ+ZsF7zwr2t9DMPgzWXWpmo8zsRKATMDzTe7/JyGxm9TMvZ3IIsD74vtIyCnjw+/x/we/yIjO7IFjfM1i3xMyGZdr/NjN7xMwWAq3N7B/B97fAzJ4O/kCQkioW85vqoUdBHkRG2FoQPN4gMrJg5eC1akRGQMroRdoWfL0ZGBg8L01kHPJqRIpyxWD9bcBd2XzeeKBr8PzvwBzgeGAxUJHIKHRLgebABcCzmd5bJfg6i2Du74xMmbbJyNgFeD54XpbIDEcVgKuAQcH6csBcoF42Obdl+v5eBc4KlisDZYLnpwGvB88vBUZlev9Q4B/B86pExhOvmOUzKmU69lkff5n7PNP7/vv95/NnfSawGfiYyJj7hwfr2xGZivXI4Pt9n8hwyNn+TIPjuRJokfmYBPt5O9M+386aGage/CzqBesPynr8Mv+OBMsfA80yHdd+2XxvdwGbiPwOXw2UD9YPAx7PtN2BwOFEhjmtHuT+CDgveN2JtOgBGgFvAfsFy2OAS8L+N6tHeI8SMeyqJJw/3L1ZxoKZ7QcMNbO/Eem6rQkcCvyc6T1fA+OCbd909wVm1hZIBv5lkWFgyxJpwWZnuJkNIjI29BVExox+w923BxmmAG2Ad4FHgpbS2+7+aT6+r3eAJ8ysHHAWMNvd/7BIF34TM+sabFeFyGQdq7K8v4KZLQi+/2+JFLaM7Z83s/pE/sPfL4fPPwPoZGa3BMvlgdrBvgBw961As3x8TwXi7jPN7Egix6MDMN/Mjgle/srdV8J/h7g8mUhhz+5nmgSsd/evg/3+HrwvmhgnEPlZrArem9sc0RmeAy4zs5uA7kTm5876vQ0JemPOAC4kMitYOyJ/aPXItN2m4Hd7lrtvDHK/BPyNyBjwacDrweanEvkD8+vge6sAaJ6CEkxFXBLBRURaKMe7+x6LzPBVPvMG7j47+I+wIzDezB4l0gp63917RvEZ/T3T+U4zOzW7jdz93xaZ8/ts4D4z+9Ddh0TzTbj7TjObRaT12R14OePjiLTkZuaxiz/cvZmZ7U9kzOtrgRHAvcDH7t7FIhcBzsrh/QZc4Ll0f5tZJSCnP0wu9H04p2tm9xP5uZD5j7MMQdGcCEw0s7eJFK/fiPxB8qdNg+/hLz9TMzs2v7kK6HUiY2V/BMzzHMZld/fvgSfN7Flgo5kdvA+ftdP/dx7ciPTm3L4voaX40TlxSQRVgF+CAt4eqJN1AzOrA2xw92eJtJKOIzIL2ElmlnGOu6KZNYjyMz8FzjOz/c2sIpGu8E/N7HBgh7u/SGTCl+Oyee+eoEcgO5OJTISQ0aqHSEHunfEeM2sQfGa23H0HcB1ws/1v2tyMKQ4vzbTpViLd4xlmAv0saMJZZNa5rPve6u7Ncnjs00VZ7j4wYx9ZXzOzU4I/SjL+gDiKSLcyQEuLzLJVisgfPZ+R8880FahhZi0y9hUcm2h8CfzNzOoF7z0om23+dCzdfSeR4/kk8H/Z7dTMOmYcayI9K2lETh28T+QPsIztDgS+AtqaWbXgHHdP4JNsdvsh0DU4x4+ZHRT87ksJpSIuieAlIMXMFgOXAMuz2aYdsNDM5hP5D/+JoGvyUmCSmS0i0u3aMJoPdPdviJwH/YrIOfLn3H0+cCzwVdCtPRi4L5u3PwMsCrpEs3oPaAt84O67g3XPAcuAb8xsCZErvXMtQEGWRUT+s38IeCD43jO/72MgObgAqjuRFvt+QbalwXKBmFkXi8zg1BqYbmZ59SZkdTwwN9PP57mMLnEip0hGEenuX0Xk9Ea2P9PgWHYHRgYXgL1Plt6anAT7vAqYErx3cjabvQz0t8gFgUcF614icnrnvRx2fTGQGvyuvABcFLSo7wMODC5gWwi098gUlQOI/MwWEmndT80m6zJgEPBe8P2/T+TiQCmhdIuZiMQdM2tH5Ja1c0KOkqPg2oIq7n5n2Fmk5NI5cRGRfDKzN4h0/Z8SdhYp2dQSFxERSVA6Jy4iIpKgVMRFREQSlIq4iIhIglIRFxERSVAq4iIiIglKRVxERCRB/T/wX5T47eI3ZQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Predict probabilities for the test data\n",
    "\n",
    "probs = value_pipe.predict_proba(X_test)[:, 1]\n",
    "\n",
    "# Get the ROC Curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, probs)\n",
    "\n",
    "# Plot ROC curve\n",
    "plt.figure(figsize=(8, 5))\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.plot(fpr, tpr)\n",
    "plt.xlabel('False Positive Rate = 1 - Specificity Score')\n",
    "plt.ylabel('True Positive Rate  = Recall Score')\n",
    "plt.title('ROC Curve')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fenotebook",
   "language": "python",
   "name": "fenotebook"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
